# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate: 
# Message:  长安官网所有车型数据抓取
import json
from untils.requestsTools import requestsTools
from untils.MysqlTools import MysqlTools
import pymongo
import os
from setting import *
import time
from _set_old import _set
import hashlib

class CaSpider:
    def __init__(self):
        self.basePath = os.getcwd() + os.sep + '../oushangHtml/old'
        if not os.path.exists(self.basePath):
            os.makedirs(self.basePath)
        self.requestsTools = requestsTools(self.basePath)
        self.requestsTools.headers = HEADERS
        self.requestsTools.cookies = COOKIES
        self.imgPath = os.getcwd() + os.sep + '../oushangImg/old'
        if not os.path.exists(self.imgPath):
            os.makedirs(self.imgPath)
        self._set = _set
        db_config = dict(
            host="192.168.10.166",
            port=3307,
            user="a",
            passwd="110",
            db="changan_new",
            charset="utf8",
        )
        self.sqlDb = MysqlTools(db_config=db_config, type='dict', auto_commit=5000)
        self.carCode_set = set()

    def get_monCol(self, col):
        monClient = pymongo.MongoClient(host='192.168.10.222', port=27017)
        monDb = monClient["changan_local"]
        monCol = monDb[col]
        return monCol

    def get_carCode(self, isOld=0):
        monCol = self.get_monCol("carCode_clean")
        data = monCol.find({"isOld": isOld})
        return list(data)

    def get_timeStr(self, len=13):
        return str(time.time()).replace('.', '')[:len]

    def urlJoin(self, url, params):
        pass

    def get_scid(self, scCode):
        url = 'http://epc.changan.com.cn:8080/record/rcd-sc/selectScIdByScCode'
        params = '?{}={}&{}={}'.format("scCode", scCode, "_", self.get_timeStr())
        url = url + params
        print("请求scid: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scCode)
        # print("debug")
        print(responseInfo.get("content"))
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", {}).get("scId", "")

    def get_groupContent_l(self, scid):
        url = 'http://epc.changan.com.cn:8080//oled/epc-tb-partconn/selectSystem'
        params = '?{}={}&{}={}'.format("scId", scid, "_", self.get_timeStr())
        url = url + params
        print("请求主组信息: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scid)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_strucContent_l(self, scid, systemId):
        url = 'http://epc.changan.com.cn:8080//oled/epc-tb-partconn/selectModule'
        params = '?{}={}&{}={}&{}={}'.format("systemId", systemId, "scId", scid, "_", self.get_timeStr())
        url = url + params
        print("请求分组信息: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scid+systemId)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_partsContent_l(self, scId, systemId, moduleId, groupnum):
        url = 'http://epc.changan.com.cn:8080//oled/epc-tb-partconn/selectPart'
        params = "?{}={}&{}={}&{}={}".format(
                                            "moduleId", moduleId,
                                            "scId", scId,
                                            "systemId", systemId,
                                                )
        url = url + params
        print("请求零件信息: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scId+systemId+moduleId+groupnum)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])


    def get_img_info(self, scid, rmmId):
        # 获取图片信息
        url = 'http://epc.changan.com.cn:8080//record/rcd-module/queryModuleImgByRmmid'
        print("请求图片信息: ", url)
        params = '?{}={}&{}={}'.format("rmmId", rmmId, "_", self.get_timeStr())
        url = url + params
        responseInfo = self.requestsTools.parseUrl(url, fileName=scid+rmmId)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_pdf(self, carCode_fina, moduleCode, pdfName):
        url = 'http://epc.changan.com.cn:8080/front/download/{}.pdf'.format(pdfName)
        print("请求pdf文件: ", url)
        # 获取filePath
        _scidPath = self.imgPath + os.sep + carCode_fina
        if not os.path.exists(_scidPath):
            os.makedirs(_scidPath)
        _modulePath = _scidPath + os.sep + moduleCode  # 分组路径
        if not os.path.exists(_modulePath):
            os.makedirs(_modulePath)
        _pdfPath = _modulePath + os.sep + pdfName + '.pdf'
        if os.path.exists(_pdfPath):
            print("pdf 已存在")
            return
        else:
            pdfResponse = self.requestsTools.getImg(url)
            if pdfResponse:
                with open(_pdfPath, 'wb') as f:
                    f.write(pdfResponse.content)

    def get_scid_oldepc(self, carCode):
        url = 'http://epc.changan.com.cn:8080//oled/epc-tb-categories/selectScAllCode'
        params = "?{}={}&{}={}".format("sc", carCode, "_", self.get_timeStr())
        url = url + params
        print("请求scid, carCode_fina 信息: ", url)
        responseInfo = self.requestsTools.parseUrl(url=url, fileName=carCode)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def _get_md5(self, str):
        '''获取md5加密电子指纹'''
        _md5 = hashlib.md5()
        _md5.update(str)
        return _md5.hexdigest()

    def save2sql(self, _dict, ctype):
        tableDict = {
            "carsinfo": "cars_info_collection_ous_old",
            "group": "cars_group_collection_ous_old",
            "struc": "cars_structure_collection_ous_old",
            "parts": "cars_part_num_ous_old"
        }
        table = tableDict[ctype]
        inSql = self.sqlDb.get_insert_sql(table, _dict)
        try:
            self.sqlDb.execute_sql(inSql)
        except Exception as e:
            print(e.__repr__())
            print(inSql)


    def save_info(self, carCode_info, carCode_fina):
        carCode = carCode_info.get("carCode")
        carCode_fina = carCode_fina
        carName = carCode_info.get("carName")
        carTail_fina = carCode_info.get("carTail_fina")
        _str = carName + carCode + carTail_fina + carCode_fina.replace("-","_").replace(".","_")+'_old'
        _dict = {
            "uid": self._get_md5(_str.encode()),
            "name": "欧尚",
            "brand": "changan",
            "brandCode": "changan",
            "condition": carName,
            "limit": carTail_fina,
            "limit2": carCode,
            "limit3": carCode_fina,
            "limit4": "",
            "label": "",
            "code": carCode_fina.replace("-","_").replace(".","_")+'_old',
            "description": "",
            "imagePath": "",
            "imgName": "",
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 2,
        }
        self.save2sql(_dict, 'carsinfo')


    def save_group(self, carCode_fina, group_num, groupContent):
        systemCh = groupContent.get("systemCh")
        _str = carCode_fina.replace("-","_").replace(".","_") + str(group_num) + systemCh
        _dict = {
            "uid": self._get_md5(_str.encode()),
            "cid": carCode_fina.replace("-","_").replace(".","_")+'_old',
            "num": str(group_num),
            "brandCode": "changan",
            "label": systemCh,
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
            # "": "",
        }
        self.save2sql(_dict, 'group')


    def save_struc(self, carCode_fina, group_num, strucContent):
        groupnum = strucContent.get("groupnum")
        subgroupCh = strucContent.get("subgroupCh")
        _str = carCode_fina.replace("-","_").replace(".","_")+'_old' + str(group_num) + groupnum.replace("-", "_") + subgroupCh
        _dict = {
            "uid": self._get_md5(_str.encode()),
            "cid": carCode_fina.replace("-","_").replace(".","_")+'_old',
            "num": str(group_num),
            "maingroup": str(group_num),
            "subgroup": groupnum.replace("-", "_"),
            "mid": groupnum.replace("-", "_"),
            "brandCode": "changan",
            "name": subgroupCh,
            "description": "",
            "model": "",
            "condition": "",
            "imgName": "",
            "imageLarge": "",
            "imgPoints": "",
            "logo": "",
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
        }
        self.save2sql(_dict, 'struc')


    def save_parts(self, carCode_fina, group_num, moduleCode, psort, partsContent):
        partcode = partsContent.get("partcode") if partsContent.get("partcode") else ""  # 备件号
        partnum = partsContent.get("partnum") if partsContent.get("partnum") else ""  # 零件号
        partnameCh = partsContent.get("partnameCh") if partsContent.get("partnameCh") else ""  # 零件名称
        qty = partsContent.get("qty")  # 零件数量
        partsequence = partsContent.get("partsequence")  # 零件pnum
        enableTime = partsContent.get("enableTime") if partsContent.get("enableTime") else ""  # 开始时间
        disableTime = partsContent.get("disableTime") if partsContent.get("disableTime") else ""  # 结束时间
        _str = carCode_fina.replace("-","_").replace(".","_")+'_old' + str(group_num) + moduleCode.replace("-", "_") + str(psort) + partnum + partcode + partnameCh
        _dict = {
            "cid": carCode_fina.replace("-","_").replace(".","_")+'_old',
            "uid": self._get_md5(_str.encode()),
            "num": group_num,
            "maingroup": group_num,
            "subgroup": moduleCode.replace("-", "_"),
            "mid": moduleCode.replace("-", "_"),
            "brandCode": "changan",
            "pnum": partsequence,
            "itid": partsequence,
            "psort": psort,
            "pid": partnum,
            "label": partnameCh,
            "remark": enableTime.replace("-", "") + "-" + disableTime.replace("-", ""),
            "quantity": str(qty) if qty else "",
            "model": partcode,
            "isreplace": "",
            "replacelabel": "",
            "rid": "",
            "condition": "",
            "img": "",
            "logo": "",
            "price": "",
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
        }
        self.save2sql(_dict, 'parts')


    def _getData(self, carCode_info):
        carCode = carCode_info.get("carCode")
        if carCode in self.carCode_set:
            return
        print(carCode)
        self.carCode_set.add(carCode)
        carCodeInfo_l = self.get_scid_oldepc(carCode)
        for carCodeInfo in carCodeInfo_l:
            scid = carCodeInfo.get("cid")
            carCode_fina = carCodeInfo.get("cname")
            # carCode_fina = carCode_info.get("carCode_fina")
            if carCode_fina in self._set:
                print("{} 已抓取过".format(carCode_fina))
                return
            print(carCode_fina)
            # continue
            # TODO 保存carsinfo表
            self.save_info(carCode_info, carCode_fina)
            continue
            # 第二次请求, 获取主组信息
            groupContent_l = self.get_groupContent_l(scid)
            group_num = 0
            for groupContent in groupContent_l:
                print(groupContent)
                systemId = groupContent.get("id", "")

                # TODO 保存group表
                group_num += 1
                self.save_group(carCode_fina, group_num, groupContent)

                if not systemId:
                    continue
                strucContent_l = self.get_strucContent_l(scid, systemId)
                for strucContent in strucContent_l:
                    print(strucContent)
                    # TODO 保存struc表
                    self.save_struc(carCode_fina, group_num, strucContent)

                    moduleId = strucContent.get("id", "")
                    groupnum = strucContent.get("groupnum", "")

                    partsContent_l = self.get_partsContent_l(scid, systemId, moduleId, groupnum)
                    psort = 0
                    for partsContent in partsContent_l:
                        partNum = partsContent.get("partnum", "")
                        compNmCn = partsContent.get("partnameCh", "")
                        psort += 1
                        self.save_parts(carCode_fina, group_num, groupnum, psort, partsContent)
                        print(partNum, compNmCn)

                    if groupnum:
                        pdfName = groupnum
                        self.get_pdf(carCode_fina, groupnum, pdfName)
            self._set.append(carCode_fina)
        self.sqlDb.commit_sql()

    def getData(self):
        # 获取新车型编号
        carCode_info_l = self.get_carCode(isOld=1)
        for carCode_info in carCode_info_l:
            # carCode_fina = carCode_info.get("carCode_fina")
            self._getData(carCode_info)
            # return

    def run(self):
        try:
            self.getData()
            with open('./_set_old.py', 'w') as s:
                s.write("_set = {}".format(self._set))
        except Exception as e:
            with open('./_set_old.py', 'w') as s:
                s.write("_set = {}".format(self._set))
            print(e.__repr__())

    # 获取欧尚数据
    def get_carsModel(self):
        url = '''http://epc.changan.com.cn:8080//oled/epc-tb-categories/selectModel'''
        params = "?{}={}&{}={}".format("brandId", "19", "_", self.get_timeStr())
        url = url + params
        responseInfo = self.requestsTools.parseUrl(url=url, fileName="19")
        data = json.loads(responseInfo["content"])["data"]
        return data

    def get_carsCode(self, cid):
        url = '''http://epc.changan.com.cn:8080/oled/epc-tb-categories/selectSc'''
        params = "?{}={}&{}={}".format("modelId", cid, "_", self.get_timeStr())
        url = url + params
        responseInfo = self.requestsTools.parseUrl(url=url, fileName=cid)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_carsStatusCode(self, carsCode):
        url = '''http://epc.changan.com.cn:8080//oled/epc-tb-categories/selectScAllCode'''
        params = "?{}={}&{}={}".format("sc", carsCode, "_", self.get_timeStr())
        url = url + params
        responseInfo = self.requestsTools.parseUrl(url=url, fileName=carsCode)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_oushangData(self):
        # 获取欧尚数据启动程序
        carsModel_l = self.get_carsModel()
        num = 0
        _set = set()
        for carsModel_info in carsModel_l:
            print(carsModel_info)
            cid = carsModel_info.get("cid")
            cname = carsModel_info.get("cname")
            carsCode_l = self.get_carsCode(cid)
            for carsCode in carsCode_l:
                print(carsCode)
                carsStatusCode_l = self.get_carsStatusCode(carsCode)
                for carsStatusCode_info in carsStatusCode_l:
                    carCode_fina = carsStatusCode_info.get("cname")
                    indextitle = carsStatusCode_info.get("indextitle")
                    print(carCode_fina)
                    num += 1
                    _set.add(carCode_fina)
        # print(num)
        # print(len(_set))
                    carCode_info = {
                        "carCode": carsCode,
                        "carName": cname,
                        "carTail_fina": indextitle.replace(carCode_fina, "").strip("| "),
                        "carCode_fina": carCode_fina,
                    }
                    self._getData(carCode_info)
        self.sqlDb.commit_sql()

if __name__ == '__main__':

    tools = CaSpider()
    tools.get_oushangData()

